# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
# -*- coding: utf-8 -*-

import json



class ProxyPoolPipeline(object):

    # 将可用的IP代理添加到代理池队列
    def process_item(self, item, spider):
        item = dict(item)
        with open('proxy.json', 'a+', encoding='utf-8') as file:
            file.seek(0)
            content = file.read()
            file.truncate(0)
            if content:
                jsonObj = json.loads(content)
            else:
                jsonObj = []
            if item not in jsonObj:
                jsonObj.append(item)
                file.write(json.dumps(jsonObj, ensure_ascii=False))

class TutorialPipeline:
    def process_item(self, item, spider):
        return item
